{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "notebookstart= time.time()\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import os\n",
    "import gc\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.linear_model import LogisticRegression, RidgeClassifier, Ridge, HuberRegressor\n",
    "from sklearn.feature_selection import RFE\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stacked = pd.read_csv('oofs/kain-train-features-v0.1.2.csv')\n",
    "test_stacked = pd.read_csv('oofs/kain-test-features-v0.1.2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('/media/limbo/Home-Credit/data/application_train.csv.zip')\n",
    "y = train['TARGET']\n",
    "\n",
    "n_train = train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features = train_stacked.values\n",
    "test_features = test_stacked.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Ridge(alpha=100, max_iter=1000, fit_intercept=False,\n",
    "                            random_state=666, normalize=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "selector = RFE(model, 105, step=2)\n",
    "selector = selector.fit(train_features, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_features = [i for i, y in enumerate(selector.ranking_) if y == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1,\n",
       " 2,\n",
       " 3,\n",
       " 4,\n",
       " 5,\n",
       " 8,\n",
       " 10,\n",
       " 12,\n",
       " 13,\n",
       " 14,\n",
       " 17,\n",
       " 18,\n",
       " 19,\n",
       " 20,\n",
       " 23,\n",
       " 24,\n",
       " 25,\n",
       " 26,\n",
       " 27,\n",
       " 28,\n",
       " 29,\n",
       " 31,\n",
       " 32,\n",
       " 34,\n",
       " 35,\n",
       " 39,\n",
       " 40,\n",
       " 41,\n",
       " 43,\n",
       " 44,\n",
       " 46,\n",
       " 49,\n",
       " 50,\n",
       " 51,\n",
       " 52,\n",
       " 53,\n",
       " 54,\n",
       " 56,\n",
       " 57,\n",
       " 58,\n",
       " 59,\n",
       " 60,\n",
       " 61,\n",
       " 62,\n",
       " 63,\n",
       " 64,\n",
       " 65,\n",
       " 67,\n",
       " 68,\n",
       " 69,\n",
       " 70,\n",
       " 74,\n",
       " 75,\n",
       " 76,\n",
       " 77,\n",
       " 79,\n",
       " 82,\n",
       " 83,\n",
       " 84,\n",
       " 85,\n",
       " 88,\n",
       " 89,\n",
       " 90,\n",
       " 92,\n",
       " 93,\n",
       " 94,\n",
       " 95,\n",
       " 97,\n",
       " 102,\n",
       " 103,\n",
       " 104,\n",
       " 105,\n",
       " 107,\n",
       " 109,\n",
       " 111,\n",
       " 115,\n",
       " 116,\n",
       " 117,\n",
       " 118,\n",
       " 119,\n",
       " 120,\n",
       " 121,\n",
       " 123,\n",
       " 127,\n",
       " 128,\n",
       " 130,\n",
       " 131,\n",
       " 133,\n",
       " 135,\n",
       " 136,\n",
       " 137,\n",
       " 139,\n",
       " 140,\n",
       " 141,\n",
       " 143,\n",
       " 144,\n",
       " 145,\n",
       " 146,\n",
       " 147,\n",
       " 148,\n",
       " 149,\n",
       " 150,\n",
       " 151,\n",
       " 152,\n",
       " 153]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(307511, 155)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Modeling Stage\n",
      "TRAIN:  [     0      1      2 ... 307505 307506 307508] TEST:  [     9     16     25 ... 307507 307509 307510]\n",
      "1   Fold AUC : 0.8039375927019463\n",
      "2   Fold AUC : 0.8040660362545016\n",
      "3   Fold AUC : 0.8040894845494969\n",
      "4   Fold AUC : 0.803591282734116\n",
      "5   Fold AUC : 0.803423169043922\n",
      "Fold AUC : 0.804043117205644\n",
      "TRAIN:  [     0      2      3 ... 307508 307509 307510] TEST:  [     1      8     17 ... 307497 307503 307504]\n",
      "1   Fold AUC : 0.8079275314782856\n",
      "2   Fold AUC : 0.8081071864675232\n",
      "3   Fold AUC : 0.8082365749394144\n",
      "4   Fold AUC : 0.8080828545486871\n",
      "5   Fold AUC : 0.8080074349642672\n",
      "Fold AUC : 0.8082833677836232\n",
      "TRAIN:  [     0      1      4 ... 307507 307509 307510] TEST:  [     2      3      6 ... 307494 307501 307508]\n",
      "1   Fold AUC : 0.8064498681103255\n",
      "2   Fold AUC : 0.8067536198567467\n",
      "3   Fold AUC : 0.8070723908650327\n",
      "4   Fold AUC : 0.8070051034318959\n",
      "5   Fold AUC : 0.8069273032279425\n",
      "Fold AUC : 0.8070849271037668\n",
      "TRAIN:  [     0      1      2 ... 307508 307509 307510] TEST:  [     7     11     13 ... 307495 307498 307506]\n",
      "1   Fold AUC : 0.8008992391479409\n",
      "2   Fold AUC : 0.8008989236810296\n",
      "3   Fold AUC : 0.8008205337383958\n",
      "4   Fold AUC : 0.8005379040645836\n",
      "5   Fold AUC : 0.8004425434360566\n",
      "Fold AUC : 0.8009162797384339\n",
      "TRAIN:  [     1      2      3 ... 307508 307509 307510] TEST:  [     0      4      5 ... 307499 307502 307505]\n",
      "1   Fold AUC : 0.8065804885096167\n",
      "2   Fold AUC : 0.8068694209387783\n",
      "3   Fold AUC : 0.8071318069379576\n",
      "4   Fold AUC : 0.8070864950481013\n",
      "5   Fold AUC : 0.8070216316012571\n",
      "Fold AUC : 0.8071602907842792\n",
      "AVERAGED AUC : 0.8054975965231493\n"
     ]
    }
   ],
   "source": [
    "aucs = []\n",
    "test_set = []\n",
    "validation_set = []\n",
    "print(\"\\nModeling Stage\")\n",
    "\n",
    "kf = KFold(n_splits=5, random_state=1002, shuffle=True)\n",
    "kf.get_n_splits(train_features)\n",
    "\n",
    "n_bagged = 6\n",
    "\n",
    "for train_index, test_index in kf.split(train_features):\n",
    "    print(\"TRAIN: \", train_index, \"TEST: \", test_index)\n",
    "\n",
    "    X = train_features\n",
    "    y_ = y.values\n",
    "    x_train, x_test = X[train_index][:, selected_features], X[test_index][:, selected_features]\n",
    "    y_train, y_test = y_[train_index], y_[test_index]\n",
    "    \n",
    "    oof_baggs = np.zeros([n_bagged , x_test.shape[0]])\n",
    "    preds_baggs = np.zeros([n_bagged , test_features.shape[0]])\n",
    "    \n",
    "    for _it in range(1, n_bagged):\n",
    "        \n",
    "        model = Ridge(alpha= min(4 **_it, 333), max_iter=1000, fit_intercept=False,\n",
    "                                random_state=666, normalize=False)\n",
    "        model.fit(x_train, y_train)\n",
    "        \n",
    "        print(_it,' ' ,'Fold AUC :', roc_auc_score(y_test, model.predict(x_test)))\n",
    "        oof_baggs[_it, :] = model.predict(x_test)\n",
    "        preds_baggs[_it, :] = model.predict(test_features[:, selected_features])\n",
    "        \n",
    "    val_preds = pd.DataFrame(oof_baggs).T\n",
    "    test_preds = pd.DataFrame(preds_baggs).T\n",
    "    \n",
    "    print('Fold AUC :', roc_auc_score(y_test, val_preds.rank(axis=0, method='min').mul(val_preds.shape[1] * [1 / val_preds.shape[1]]).sum(1) / val_preds.shape[0]))\n",
    "    aucs.append(roc_auc_score(y_test, val_preds.rank(axis=0, method='min').mul(val_preds.shape[1] * [1 / val_preds.shape[1]]).sum(1) / val_preds.shape[0]))\n",
    "   \n",
    "    test_set.append(test_preds.rank(axis=0, method='min').mul(test_preds.shape[1] * [1 / test_preds.shape[1]]).sum(1) / test_preds.shape[0])\n",
    "    gc.collect()\n",
    "        \n",
    "print('AVERAGED AUC :', np.mean(aucs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds = pd.DataFrame(test_set).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame({'folds': range(5), 'aucs': aucs}).to_csv('folds/ridge-folds.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100001</td>\n",
       "      <td>0.356208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100005</td>\n",
       "      <td>0.835775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100013</td>\n",
       "      <td>0.407283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100028</td>\n",
       "      <td>0.379887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100038</td>\n",
       "      <td>0.787982</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR    TARGET\n",
       "0      100001  0.356208\n",
       "1      100005  0.835775\n",
       "2      100013  0.407283\n",
       "3      100028  0.379887\n",
       "4      100038  0.787982"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "y_hat = test_preds.rank(axis=0, method='min').mul(test_preds.shape[1] * [1 / test_preds.shape[1]]).sum(1) / test_preds.shape[0] \n",
    "    \n",
    "\n",
    "\n",
    "# In[82]:\n",
    "\n",
    "\n",
    "y_hat.shape\n",
    "\n",
    "\n",
    "# In[83]:\n",
    "\n",
    "\n",
    "y_hat.head(10)\n",
    "\n",
    "\n",
    "# In[84]:\n",
    "\n",
    "\n",
    "sampl_sub = pd.read_csv('../data/sample_submission.csv')\n",
    "\n",
    "\n",
    "# In[86]:\n",
    "\n",
    "\n",
    "sampl_sub['TARGET'] = y_hat.clip(0, 1).values\n",
    "\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "\n",
    "sampl_sub.to_csv(\"ridge-stack-submission.csv\", index=False)\n",
    "\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "\n",
    "sampl_sub.head()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
